package com.liqw.crawler.crawler.processor;

import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.selector.Selectable;

import java.util.List;

@Component
@Slf4j
public class JdFapaiPageProcessor extends AbstractPageProcessor implements InitializingBean {
    @Override
    public String getType() {
        return "jd";
    }

    private Site site = Site.me()
            .setCharset("UTF-8")
            .setRetryTimes(3)
            .setSleepTime(1000)
            .setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36")
            .addHeader("Referer", "strict-origin-when-cross-origin") // 全局 Referer
            ;

    @Override
    public void process(Page page) {
        Selectable url = page.getUrl();
        log.info("正在解析url:{}", url.toString());
        log.info("page.getHtml()={}", page.getHtml());

        List<Selectable> nodes = page.getHtml().css("body > div.sf-wrap > div.sf-content > div.sf-item-list > ul").nodes();
        for (Selectable node : nodes) {
            String title = node.xpath("a/text()").get();
            System.out.println(title);
        }

        // 提取页面标题
        page.putField("title", page.getHtml().xpath("//title/text()").get());

        // 提取页面中的所有链接
        //page.addTargetRequests(page.getHtml().links().regex("(https://.*)").all());

        // 示例：提取特定内容（这里以CSDN博客为例）
        //page.putField("content", page.getHtml().xpath("//div[@class='markdown_views']").get());
    }

    @Override
    public Site getSite() {
        return site;
    }

    @Override
    public void afterPropertiesSet() throws Exception {
        AbstractPageProcessorManager.addPageProcessor(this);
    }
}